{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 3 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.9416, -0.1710, -1.5323,  0.4573,  0.0620]],\n",
       "       grad_fn=<EmbeddingBackward>)"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeds = nn.Embedding(2,5) # 构建了一张表格 是索引到embedding的映射\n",
    "word_to_ix = {'hello':0,'world':1}\n",
    "lookup_tensor = torch.tensor([word_to_ix['hello']],dtype = torch.long)\n",
    "embeds(lookup_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(['When', 'forty'], 'winters'), (['forty', 'winters'], 'shall'), (['winters', 'shall'], 'besiege')]\n"
     ]
    }
   ],
   "source": [
    "test_sentence = \"\"\"When forty winters shall besiege thy brow,\n",
    "And dig deep trenches in thy beauty's field,\n",
    "Thy youth's proud livery so gazed on now,\n",
    "Will be a totter'd weed of small worth held:\n",
    "Then being asked, where all thy beauty lies,\n",
    "Where all the treasure of thy lusty days;\n",
    "To say, within thine own deep sunken eyes,\n",
    "Were an all-eating shame, and thriftless praise.\n",
    "How much more praise deserv'd thy beauty's use,\n",
    "If thou couldst answer 'This fair child of mine\n",
    "Shall sum my count, and make my old excuse,'\n",
    "Proving his beauty by succession thine!\n",
    "This were to be new made when thou art old,\n",
    "And see thy blood warm when thou feel'st it cold.\"\"\".split()\n",
    "trigrams = [([test_sentence[i],test_sentence[i+1]],test_sentence[i+2]) for i in range(len(test_sentence)-2)]\n",
    "print(trigrams[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "83"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# index\n",
    "vocab = set(test_sentence)\n",
    "word_to_ix = {word:i for i,word in enumerate(vocab)}\n",
    "word_to_ix['When']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class NGramLanguageModeler(nn.Module):\n",
    "    def __init__(self,vocab_size,embedding_dim):\n",
    "        super(NGramLanguageModeler,self).__init__()\n",
    "        self.embeddings = nn.Embedding(vocab_size,embedding_dim)\n",
    "        self.linear1 = nn.Linear(2*embedding_dim,128)\n",
    "        self.linear2 = nn.Linear(128,vocab_size)\n",
    "    def forward(self,inputs):\n",
    "        embeds = self.embeddings(inputs).view(1,-1)\n",
    "        out = F.relu(self.linear1(embeds))\n",
    "        out = self.linear2(out)\n",
    "        log_probs = F.log_softmax(out,dim=1)\n",
    "        return log_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss_function = nn.NLLLoss()\n",
    "model = NGramLanguageModeler(len(vocab),10,)\n",
    "optimizer = optim.SGD(model.parameters(),lr=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:1 loss:507.4855\n",
      "Epoch:2 loss:505.1290\n",
      "Epoch:3 loss:502.7850\n",
      "Epoch:4 loss:500.4534\n",
      "Epoch:5 loss:498.1322\n",
      "Epoch:6 loss:495.8198\n",
      "Epoch:7 loss:493.5148\n",
      "Epoch:8 loss:491.2182\n",
      "Epoch:9 loss:488.9304\n",
      "Epoch:10 loss:486.6491\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(10):\n",
    "    epoch_loss = 0\n",
    "    for tri in trigrams:\n",
    "        # 会累乘梯度 所以得归零\n",
    "        model.zero_grad()\n",
    "        context_idxs = torch.tensor([word_to_ix[w] for w in tri[0]],dtype=torch.long)\n",
    "        target = torch.tensor([word_to_ix[tri[1]]],dtype=torch.long)\n",
    "        log_probs = model(context_idxs)\n",
    "        loss = loss_function(log_probs,target)\n",
    "        # 反向传播更新梯度\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        # 为了观察更加\n",
    "        epoch_loss += loss.item()\n",
    "    print('Epoch:%d loss:%.4f'%(epoch+1,epoch_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# word2vec → word_embedding\n",
    "# 实例网址 https://www.jianshu.com/p/ce630c198762\n",
    "# 背后数学原理 https://blog.csdn.net/itplus/article/details/37969519\n",
    "# python 工具包的使用\n",
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "?Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_sentence = \"\"\"When forty winters shall besiege thy brow,\n",
    "And dig deep trenches in thy beauty's field,\n",
    "Thy youth's proud livery so gazed on now,\n",
    "Will be a totter'd weed of small worth held:\n",
    "Then being asked, where all thy beauty lies,\n",
    "Where all the treasure of thy lusty days;\n",
    "To say, within thine own deep sunken eyes,\n",
    "Were an all-eating shame, and thriftless praise.\n",
    "How much more praise deserv'd thy beauty's use,\n",
    "If thou couldst answer 'This fair child of mine\n",
    "Shall sum my count, and make my old excuse,'\n",
    "Proving his beauty by succession thine!\n",
    "This were to be new made when thou art old,\n",
    "And see thy blood warm when thou feel'st it cold.\"\"\".lower().split()\n",
    "model = Word2Vec([test_sentence],window=5,min_count=0,size=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.01793834, -0.01433753,  0.01907553,  0.01704289, -0.00996032,\n",
       "       -0.0219532 ,  0.00630023, -0.00810397, -0.00496258, -0.0151518 ,\n",
       "        0.00218972, -0.00564141, -0.02158547,  0.00682365, -0.01145782,\n",
       "       -0.01581302,  0.02161076, -0.01857579,  0.00068021, -0.02373642],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.get_vector('when')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('his', 0.4808335304260254),\n",
       " ('trenches', 0.4757190942764282),\n",
       " ('and', 0.47177618741989136),\n",
       " ('livery', 0.3858475685119629),\n",
       " ('in', 0.38496193289756775)]"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.similar_by_word('beauty',topn=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 中文 Jieba分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
